import requests
from bs4 import BeautifulSoup
import jieba.posseg as pseg


# 笔趣阁网页获取章节小说
def get_chapter_novel(url):
    # 获取HTML文本
    r = requests.get(url)
    r.raise_for_status()  # 如果状态不是200，引发异常
    r.encoding = 'utf-8'  # 无论原来用什么编码，都改成utf-8
    html_text = r.text
    # 提取小说内容
    soup = BeautifulSoup(html_text, 'lxml')
    print("正在爬取： " + url)
    content_div = soup.find('div', id='content')
    if content_div:
        p_tags = content_div.find_all('p')
        # 跳过第一个p标签内容提取剩余的 p 标签内容并拼接
        novel_text = ''.join([p.text for p in p_tags[1:]])
    else:
        print(url + ": 该章节爬取失败: 未找到<div id=content>")
        return url + "该章节爬取失败"
    novel_text = novel_text.replace('\n\n', '').replace(' ', '')
    return novel_text

# 保存
def save_novel(root, path, text):
    try:
        import os
        if not os.path.exists(root):
            os.makedirs(root)
        if not os.path.exists(path):
            with open(path, "wb") as f:
                f.write(text.encode())
                print("文件保存成功")
        else:
            with open(path, "ab") as f:
                f.write(text.encode())
                print("文件追加成功")
    except Exception as e:
        print("保存过程异常:", e)

# 对提取的词进行过滤 + 统计，返回字典
def name_filter(words):
    counts = {}
    # 排除的错误姓名
    excludes = {"福生玄", "毛巾", "黄铜", "张开", "莫雷蒂", "墨水瓶", "盏灯", "小丑", "占卜师"}
    for word in words:
        if word == "克莱恩":
            rword = "克莱恩.莫雷蒂"
        elif word == '':
            rword = word
        else:
            rword = word
        counts[rword] = counts.get(rword, 0) + 1
    for word in excludes:
        if word in counts:
            del counts[word]
    return counts

# 获取章数
n = 3
# 保存路径
root = './novel/'
path = root + "3.txt"

for i in range(n):
    url = "https://www.biquke.vip/book/209/{}.html".format(i + 139978)  # 诡秘之主
    # url = "https://www.biquke.vip/book/200/{}.html".format(i + 133810) # 遮天
    novel_text = get_chapter_novel(url)
    # 保存这一章小说
    save_novel(root, path, novel_text + "\n")
    # 提取人物姓名
    words = pseg.cut(novel_text)
    words = [word.word for word in words if word.flag == 'nr' and len(word.word) > 1]

